# -*- coding:UTF-8 -*-
import random
import re
from bs4 import BeautifulSoup
import requests
import threading
import time

from spider_xue.account import set_account_use_not
from spider_xue.utils import send_msg
from spider_xue.settings import URL1, COOKIES
from spider_xue.ip import headers_list
from db import *


class NewIP():
    def __init__(self):
        pass

    def save(self):
        with open("spider_xue/new_ip.txt", 'r') as f:
            g = f.readlines()
            p = []
            ts = []
            for i in g:
                if not i.strip():
                    continue
                p.append(i.strip())
                if len(p) > 20:
                    try:
                        t1 = threading.Thread(target=test_cookies, args=(
                            p, new_ip_txt_db, fail_ip_db, ))
                        t1.start()
                        p = []
                        ts.append(t1)
                    except Exception as e:
                        print(e)
                        continue
            if p:
                test_cookies(p, new_ip_txt_db, fail_ip_db)
            for ttt in ts:
                ttt.join()

    def save_dirty(self):
        with open("news/new_ip.txt", 'r') as f:
            g = f.readlines()
            p = []
            ts = []
            for i in g:
                if not i.strip():
                    continue
                p.append(i.strip())
                if len(p) > 20:
                    try:
                        t1 = threading.Thread(target=test_cookies, args=(
                            p, dirty_ip_db, fail_ip_db, ))
                        t1.start()
                        p = []
                        ts.append(t1)
                    except Exception as e:
                        print(e)
                        continue
            if p:
                test_cookies(p, new_ip_txt_db, fail_ip_db)
            for ttt in ts:
                ttt.join()

    def save_req(self, g):
        p = []
        ts = []
        for i in g:
            if not i.strip() and self.failip.find_one({"ip": i.strip()}):
                continue
            p.append(i.strip())
            if len(p) > 20:
                try:
                    t1 = threading.Thread(
                        target=test_cookies, args=(p, new_ip_db, fail_ip_db, ))
                    t1.start()
                    p = []
                    ts.append(t1)
                except Exception as e:
                    print(e)
                    continue
        if p:
            test_cookies(p, new_ip_db, fail_ip_db)
        for ttt in ts:
            ttt.join()

    def get_req(self, count1):
        hd = {
            'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', 'accept-encoding': 'gzip, deflate, br',
            'sec-ch-ua': '"Microsoft Edge";v="93", " Not;A Brand";v="99", "Chromium";v="93"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': "Windows",
            'sec-fetch-dest': 'document',
            'sec-fetch-mode': 'navigate',
            'sec-fetch-site': 'same-site',
            'host': 'ip.jiangxianli.com',
            'dnt': '1',
            'cookie': "Hm_lvt_b72418f3b1d81bbcf8f99e6eb5d4e0c3=1636939272,1637629562,1637717260,1637802931; Hm_lpvt_b72418f3b1d81bbcf8f99e6eb5d4e0c3=1637802979",
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36 Edg/94.0.992.31'
        }
        pt = '(\d{1,4}\.\d.*?)@'
        while True:
            try:
                url = f'http://ip.jiangxianli.com/blog/{count1}.html'
                res = requests.get(url, headers=hd, timeout=5, verify=False)
                if '1970年01' in res.text or res.status_code != 200:
                    break
                soup = BeautifulSoup(res.text, "lxml")
                div = soup.find_all("p")
                dsds = re.findall(pt, div[0].text)
                ret = [tdss for tdss in dsds]
                self.save_req(ret)
                time.sleep(1)
                count1 += 1
            except Exception as e:
                print(count1, 'fail', e)
                return count1

        return count1

    def get_one(self):
        x = new_ip_db.find_one({"status": 1})
        if x:
            new_ip_db.update_one(
                {"ip": x['ip']}, {"$set": {"status": 0}}, upsert=True)
            return x

    def clean(self):
        new_ip_db.delete_many({"speed": {"$gt": 13}})

    def delete_one(self, iiiip):
        if iiiip['count'] == 0:
            new_ip_db.delete_many({"ip": iiiip['ip']})
        else:
            new_ip_db.update_one({"ip": iiiip['ip']}, {
                                 "$set": {"status": 1, "count": iiiip['count']-1}}, upsert=True)

    def set_back_one(self, iiiip):
        new_ip_db.update_one(
            {"ip": iiiip}, {"$set": {"status": 1}}, upsert=True)


def clean_db_ip():
    all_ip = new_ip_db.find()
    l = [x['ip'] for x in all_ip]
    # for i in all_ip:
    ff = test_cookies_duplicate(l, new_ip_db, fail_ip_db)
    for i in ff:
        dirty_ip_db.update_one(
            {"ip": i}, {"$set": {'ip': i, 'count': 3, 'status': 1}}, upsert=True)


def clean_db_txt_ip():
    all_ip = new_ip_txt_db.find()
    l = [x['ip'] for x in all_ip]
    # for i in l:
    #     print(i)
    ff = test_cookies_duplicate(l, new_ip_txt_db, fail_ip_db)
    for i in ff:
        dirty_ip_db.update_one(
            {"ip": i}, {"$set": {'ip': i, 'count': 3, 'status': 1}}, upsert=True)


def clean_dirty_db_ip():
    all_ip = dirty_ip_db.find()
    l = [x['ip'] for x in all_ip]
    # for i in l:
    #     print(i)
    ff = test_cookies_duplicate(l, dirty_ip_db, fail_ip_db)
    # for i in range(1):
    #     ff = test_cookies_duplicate(ff, dirty_ip_db, fail_ip_db)


def check_duplicate_ip():
    all_ip = new_ip_db.find()
    txt_ip = new_ip_txt_db.find()
    ditry_ip = dirty_ip_db.find()

    all_ip = [x['ip'] for x in all_ip]
    txt_ip = [x['ip'] for x in txt_ip]
    ditry_ip = [x['ip'] for x in ditry_ip]

    t1 = list(set(all_ip) & set(txt_ip))
    print(len(t1))
    # for i in t1:
    #     new_ip_txt_db.delete_one({'ip': i})

    t2 = list(set(all_ip) & set(ditry_ip))
    print(len(t2))
    for i in t2:
        dirty_ip_db.delete_one({'ip': i})

    t3 = list(set(txt_ip) & set(ditry_ip))
    print(len(t3))
    for i in t3:
        dirty_ip_db.delete_one({'ip': i})


def test_cookies(ips, newip, failip):
    l1 = []
    l2 = []
    print("starting................")
    fail = []
    for i in ips:
        if len(i) < 6:
            continue
        try:
            # and not failip.find_one({"ip": i}):
            if not newip.find_one({"ip": i}):
                send_req(i, "SH600096")
                l1.append(i)
        except Exception as e:
            fail.append(i)
            print(i, "fail111111111")
            newip.delete_one({'ip': i})
            failip.update_one({"ip": i}, {"$set": {'ip': i}}, upsert=True)
            continue
    for i in l1:
        try:
            send_req(i, "SZ002184")
            l2.append(i)

        except Exception as e:
            print(i, "fail22222222222")
            fail.append(i)
            newip.delete_one({'ip': i})
            failip.update_one({"ip": i}, {"$set": {'ip': i}}, upsert=True)
            continue
    for i in l2:
        ip_str = {"ip": i, 'count': 3, 'status': 1}
        try:
            send_req(i, "SZ002415")
            print(i, "success................................")
            newip.update_one({"ip": i}, {"$set": ip_str}, upsert=True)
        except Exception as e:
            print(i, "fail33333333333333")
            fail.append(i)
            dirty_ip_db.update_one({"ip": i}, {"$set": ip_str}, upsert=True)
            continue

    print("ending................")
    return fail


def test_cookies_duplicate(ips, newip, failip):
    l1 = []
    l2 = []
    print("starting................")
    fail = []
    for i in ips:
        if len(i) < 6:
            continue
        try:
            send_req(i, "SH600096")
            l1.append(i)

        except Exception as e:
            fail.append(i)
            print(i, "fail111111111")
            newip.delete_one({'ip': i})
            failip.update_one({"ip": i}, {"$set": {'ip': i}}, upsert=True)
            continue
    for i in l1:
        try:
            send_req(i, "SZ002184")
            l2.append(i)
        except Exception as e:
            print(i, "fail22222222222")
            fail.append(i)
            newip.delete_one({'ip': i})
            failip.update_one({"ip": i}, {"$set": {'ip': i}}, upsert=True)
            continue
    for i in l2:
        ip_str = {"ip": i, 'count': 3, 'status': 1}
        try:
            send_req(i, "SZ002415")
            print(i, "success................................")
            newip.update_one({"ip": i}, {"$set": ip_str}, upsert=True)
        except Exception as e:
            print(i, "fail33333333333333")
            fail.append(i)
            dirty_ip_db.update_one({"ip": i}, {"$set": ip_str}, upsert=True)
            continue
    print("ending................")
    return fail


def send_req(i, sy):
    tt = str(int(time.time())) + "000"
    ss = requests.Session()
    proxies = {"http": "http://%s" % i}
    # proxies = {"http": "http://45.179.164.9:80"}
    # proxies = {"http": "http://39.105.205.245:2222"}
    ss.headers = {
        "user-agent": random.choice(headers_list), "cookie": account_db.find_one()['cookies']}
    html = ss.get(URL1.format(sy, tt, 142), proxies=proxies, timeout=3)
    if html.status_code != 200:
        raise ValueError("not 200")
    if not html.json():
        print("jiexishibai ")
        raise ValueError("not jiexi")
    if html.json()['data']['symbol'] != sy:
        print("not !!!symbol ")
        raise ValueError("not symbol")


new_ip_obj = NewIP()


if __name__ == "__main__":

    c = new_ip_obj.save()
    # print(c)
    # send_req("120.196.112.6:3128")
